--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Node / ReticulumProjects / MeshChatX.git / files / tests / backend / test_sqlite_landlock_temp_store.py
Displaying Raw • Download
tests/backend/test_sqlite_landlock_temp_store.py c0b0ddffd735a0d007c1be9a5e8563ec42fdafc0 (c0b0ddff) Text, 9.29 KB
T8b949e# SPDX-License-Identifier: 0BSD
T8b949e"""Regression: worker-thread SQLite connections must use MEMORY temp under Landlock."""
Tff7b72import T7ee787subprocess
Tff7b72import T7ee787sys
Tff7b72import T7ee787textwrap
Tff7b72from T7ee787concurrentT7ee787.T7ee787futures Tff7b72import Te6edf3ThreadPoolExecutor
Tff7b72from T7ee787pathlib Tff7b72import Te6edf3Path
Tff7b72import T7ee787pytest
Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787databaseT7ee787.T7ee787provider Tff7b72import Te6edf3DatabaseProvider
Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787landlock_sandbox Tff7b72import Te6edf3landlock_kernel_supported
Tf0883e@pytestTff7b72.Td2a8fffixtureTb4b4b4(Te6edf3autouseTff7b72=Tff7b72TrueTb4b4b4)
Tff7b72def Td2a8ffreset_providerTb4b4b4(Tb4b4b4)Tb4b4b4:
Tff7b72if Te6edf3DatabaseProviderTff7b72.Td2a8ff_instance Tff7b72is Tff7b72not Tff7b72NoneTb4b4b4:
Te6edf3DatabaseProviderTff7b72.Td2a8ff_instanceTff7b72.Td2a8ffclose_allTb4b4b4(Tb4b4b4)
Te6edf3DatabaseProviderTff7b72.Td2a8ff_instance Tff7b72= Tff7b72None
Tff7b72yield
Tff7b72if Te6edf3DatabaseProviderTff7b72.Td2a8ff_instance Tff7b72is Tff7b72not Tff7b72NoneTb4b4b4:
Te6edf3DatabaseProviderTff7b72.Td2a8ff_instanceTff7b72.Td2a8ffclose_allTb4b4b4(Tb4b4b4)
Te6edf3DatabaseProviderTff7b72.Td2a8ff_instance Tff7b72= Tff7b72None
Tff7b72def Td2a8fftest_provider_configures_temp_store_memory_on_new_connectionsTb4b4b4(Te6edf3tmp_pathTb4b4b4)Tb4b4b4:
Te6edf3db_path Tff7b72= Tffa657strTb4b4b4(Te6edf3tmp_path Tff7b72/ Ta5d6ff"Ta5d6ffdatabase.dbTa5d6ff"Tb4b4b4)
Te6edf3provider Tff7b72= Te6edf3DatabaseProviderTff7b72.Td2a8ffget_instanceTb4b4b4(Te6edf3db_pathTb4b4b4)
Te6edf3conn Tff7b72= Te6edf3providerTff7b72.Td2a8ffconnection
Te6edf3mode Tff7b72= Te6edf3connTff7b72.Td2a8ffexecuteTb4b4b4(Ta5d6ff"Ta5d6ffPRAGMA temp_storeTa5d6ff"Tb4b4b4)Tff7b72.Td2a8fffetchoneTb4b4b4(Tb4b4b4)Tb4b4b4[T79c0ff0Tb4b4b4]
T8b949e# SQLite returns 0=DEFAULT, 1=FILE, 2=MEMORY
Tff7b72assert Tffa657intTb4b4b4(Te6edf3modeTb4b4b4) Tff7b72== T79c0ff2
Tff7b72def Td2a8fftest_provider_memory_pressure_prefers_file_tempTb4b4b4(Te6edf3tmp_pathTb4b4b4)Tb4b4b4:
Te6edf3db_path Tff7b72= Tffa657strTb4b4b4(Te6edf3tmp_path Tff7b72/ Ta5d6ff"Ta5d6ffdatabase.dbTa5d6ff"Tb4b4b4)
Te6edf3provider Tff7b72= Te6edf3DatabaseProviderTff7b72.Td2a8ffget_instanceTb4b4b4(Te6edf3db_pathTb4b4b4)
Te6edf3providerTff7b72.Td2a8ffprefer_temp_store_file Tff7b72= Tff7b72True
Te6edf3providerTff7b72.Td2a8ffcloseTb4b4b4(Tb4b4b4)
Te6edf3conn Tff7b72= Te6edf3providerTff7b72.Td2a8ffconnection
Te6edf3mode Tff7b72= Te6edf3connTff7b72.Td2a8ffexecuteTb4b4b4(Ta5d6ff"Ta5d6ffPRAGMA temp_storeTa5d6ff"Tb4b4b4)Tff7b72.Td2a8fffetchoneTb4b4b4(Tb4b4b4)Tb4b4b4[T79c0ff0Tb4b4b4]
Tff7b72assert Tffa657intTb4b4b4(Te6edf3modeTb4b4b4) Tff7b72== T79c0ff1
Tff7b72def Td2a8fftest_worker_threads_inherit_memory_temp_storeTb4b4b4(Te6edf3tmp_pathTb4b4b4)Tb4b4b4:
Te6edf3db_path Tff7b72= Tffa657strTb4b4b4(Te6edf3tmp_path Tff7b72/ Ta5d6ff"Ta5d6ffdatabase.dbTa5d6ff"Tb4b4b4)
Te6edf3provider Tff7b72= Te6edf3DatabaseProviderTff7b72.Td2a8ffget_instanceTb4b4b4(Te6edf3db_pathTb4b4b4)
Te6edf3modesTb4b4b4: Tffa657listTb4b4b4[Tffa657intTb4b4b4] Tff7b72= Tb4b4b4[Tb4b4b4]
Te6edf3errorsTb4b4b4: Tffa657listTb4b4b4[Tffa657strTb4b4b4] Tff7b72= Tb4b4b4[Tb4b4b4]
Tff7b72def Td2a8ffworkerTb4b4b4(Te6edf3_iTb4b4b4: Tffa657intTb4b4b4) Tff7b72-Tff7b72> Tff7b72NoneTb4b4b4:
Tff7b72tryTb4b4b4:
Te6edf3conn Tff7b72= Te6edf3providerTff7b72.Td2a8ffconnection
Te6edf3modesTff7b72.Td2a8ffappendTb4b4b4(Tffa657intTb4b4b4(Te6edf3connTff7b72.Td2a8ffexecuteTb4b4b4(Ta5d6ff"Ta5d6ffPRAGMA temp_storeTa5d6ff"Tb4b4b4)Tff7b72.Td2a8fffetchoneTb4b4b4(Tb4b4b4)Tb4b4b4[T79c0ff0Tb4b4b4]Tb4b4b4)Tb4b4b4)
Tff7b72except Tf85149Exception Tff7b72as Te6edf3excTb4b4b4:
Te6edf3errorsTff7b72.Td2a8ffappendTb4b4b4(Tffa657strTb4b4b4(Te6edf3excTb4b4b4)Tb4b4b4)
Tff7b72with Te6edf3ThreadPoolExecutorTb4b4b4(Te6edf3max_workersTff7b72=T79c0ff8Tb4b4b4) Tff7b72as Te6edf3poolTb4b4b4:
Tffa657listTb4b4b4(Te6edf3poolTff7b72.Td2a8ffmapTb4b4b4(Te6edf3workerTb4b4b4, Tffa657rangeTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4)Tb4b4b4)
Tff7b72assert Tff7b72not Te6edf3errors
Tff7b72assert Te6edf3modes
Tff7b72assert Tffa657allTb4b4b4(Te6edf3mode Tff7b72== T79c0ff2 Tff7b72for Te6edf3mode Tff7b72in Te6edf3modesTb4b4b4)
Tf0883e@pytestTff7b72.Td2a8ffmarkTff7b72.Td2a8ffskipifTb4b4b4(
Tff7b72not Te6edf3landlock_kernel_supportedTb4b4b4(Tb4b4b4)Tb4b4b4,
Te6edf3reasonTff7b72=Ta5d6ff"Ta5d6ffLandlock not available on this kernelTa5d6ff"Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8fftest_landlock_worker_heavy_query_ok_with_memory_tempTb4b4b4(Tb4b4b4)Tb4b4b4:
T8b949e"""Worker-thread heavy conversation queries must work under Landlock.
Runs in a subprocess because Landlock can only restrict a process once.
"""
Te6edf3script Tff7b72= Te6edf3textwrapTff7b72.Td2a8ffdedentTb4b4b4(
Ta5d6ffrT8b949e"""
import os, sqlite3, sys, tempfile, time
from concurrent.futures import ThreadPoolExecutor
from meshchatx.src.backend.database.provider import DatabaseProvider
from meshchatx.src.backend.landlock_sandbox import apply_landlock_sandbox
td = tempfile.mkdtemp(prefix="ll_sqlite_reg_")
storage = os.path.join(td, "storage")
os.makedirs(storage)
os.environ["MESHCHAT_LANDLOCK"] = "1"
db = os.path.join(storage, "database.db")
conn = sqlite3.connect(db)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute(
"CREATE TABLE lxmf_messages ("
"id INTEGER PRIMARY KEY, peer_hash TEXT, content TEXT, fields TEXT, "
"title TEXT, timestamp REAL, is_incoming INT, state TEXT)"
)
big_content = "x" * 200000
big_fields = '{"image":{"image_bytes":"' + ("A" * 40000) + '"}}'
for i in range(80):
conn.execute(
"INSERT INTO lxmf_messages VALUES (?,?,?,?,?,?,?,?)",
(i, f"peer{i%16}", big_content, big_fields, "t", time.time(), 1, "delivered"),
)
conn.commit()
conn.close()
ok = apply_landlock_sandbox(
storage_dir=storage,
reticulum_config_dir=storage,
log_dir=storage,
)
if not ok:
print("LANDLOCK_NOT_APPLIED")
sys.exit(2)
heavy = '''
SELECT m1.id, substr(COALESCE(m1.content,''),1,240) AS content,
CASE WHEN instr(m1.fields,'"image"')>0 THEN 1 ELSE 0 END AS has_image
FROM lxmf_messages m1
INNER JOIN (
SELECT peer_hash, MAX(id) AS max_id FROM lxmf_messages
WHERE peer_hash IS NOT NULL GROUP BY peer_hash
) m2 ON m1.peer_hash=m2.peer_hash AND m1.id=m2.max_id
GROUP BY m1.peer_hash ORDER BY m1.id DESC LIMIT 50
'''
DatabaseProvider._instance = None
provider = DatabaseProvider.get_instance(db)
mem_errors = []
def mem_worker(_i):
try:
rows = provider.fetchall(heavy)
if not rows:
mem_errors.append("empty")
except Exception as e:
mem_errors.append(str(e))
with ThreadPoolExecutor(max_workers=8) as ex:
list(ex.map(mem_worker, range(24)))
print("MEM_ERRORS", len(mem_errors))
if mem_errors:
print(mem_errors[:3])
sys.exit(4)
sys.exit(0)
"""Tb4b4b4,
Tb4b4b4)
Te6edf3result Tff7b72= Te6edf3subprocessTff7b72.Td2a8ffrunTb4b4b4(
Tb4b4b4[Te6edf3sysTff7b72.Td2a8ffexecutableTb4b4b4, Ta5d6ff"Ta5d6ff-cTa5d6ff"Tb4b4b4, Te6edf3scriptTb4b4b4]Tb4b4b4,
Te6edf3cwdTff7b72=Tffa657strTb4b4b4(Te6edf3PathTb4b4b4(Tff7b72__file__Tb4b4b4)Tff7b72.Td2a8ffresolveTb4b4b4(Tb4b4b4)Tff7b72.Td2a8ffparentsTb4b4b4[T79c0ff2Tb4b4b4]Tb4b4b4)Tb4b4b4,
Te6edf3capture_outputTff7b72=Tff7b72TrueTb4b4b4,
Te6edf3textTff7b72=Tff7b72TrueTb4b4b4,
Te6edf3timeoutTff7b72=T79c0ff60Tb4b4b4,
Te6edf3checkTff7b72=Tff7b72FalseTb4b4b4,
Tb4b4b4)
Tff7b72if Ta5d6ff"Ta5d6ffLANDLOCK_NOT_APPLIEDTa5d6ff" Tff7b72in Te6edf3resultTff7b72.Td2a8ffstdoutTb4b4b4:
Te6edf3pytestTff7b72.Td2a8ffskipTb4b4b4(Ta5d6ff"Ta5d6ffLandlock could not be applied in this environmentTa5d6ff"Tb4b4b4)
Tff7b72assert Te6edf3resultTff7b72.Td2a8ffreturncode Tff7b72== T79c0ff0Tb4b4b4, Tb4b4b4(
Ta5d6fffTa5d6ff"Ta5d6ffstdout=Tffd700{Te6edf3resultTff7b72.Td2a8ffstdoutTffd700!r}Ta5d6ff stderr=Tffd700{Te6edf3resultTff7b72.Td2a8ffstderrTffd700!r}Ta5d6ff code=Tffd700{Te6edf3resultTff7b72.Td2a8ffreturncodeTffd700}Ta5d6ff"
Tb4b4b4)
Tff7b72assert Ta5d6ff"Ta5d6ffMEM_ERRORS 0Ta5d6ff" Tff7b72in Te6edf3resultTff7b72.Td2a8ffstdout
Tf0883e@pytestTff7b72.Td2a8ffmarkTff7b72.Td2a8ffskipifTb4b4b4(
Tff7b72not Te6edf3landlock_kernel_supportedTb4b4b4(Tb4b4b4)Tb4b4b4,
Te6edf3reasonTff7b72=Ta5d6ff"Ta5d6ffLandlock not available on this kernelTa5d6ff"Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8fftest_landlock_memory_pressure_keeps_memory_temp_and_queries_okTb4b4b4(Tb4b4b4)Tb4b4b4:
T8b949e"""Memory-pressure must not switch to FILE temp under Landlock."""
Te6edf3script Tff7b72= Te6edf3textwrapTff7b72.Td2a8ffdedentTb4b4b4(
Ta5d6ffrT8b949e"""
import os, sqlite3, sys, tempfile, time
from concurrent.futures import ThreadPoolExecutor
from meshchatx.src.backend.database import Database
from meshchatx.src.backend.database.provider import DatabaseProvider
from meshchatx.src.backend.landlock_sandbox import apply_landlock_sandbox
td = tempfile.mkdtemp(prefix="ll_pressure_")
storage = os.path.join(td, "storage")
os.makedirs(storage)
os.environ["MESHCHAT_LANDLOCK"] = "1"
db_path = os.path.join(storage, "database.db")
conn = sqlite3.connect(db_path)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute(
"CREATE TABLE lxmf_messages ("
"id INTEGER PRIMARY KEY, peer_hash TEXT, content TEXT, fields TEXT, "
"title TEXT, timestamp REAL, is_incoming INT, state TEXT)"
)
big_content = "x" * 120000
big_fields = '{"image":{"image_bytes":"' + ("A" * 60000) + '"}}'
for i in range(200):
conn.execute(
"INSERT INTO lxmf_messages VALUES (?,?,?,?,?,?,?,?)",
(i, f"peer{i%40}", big_content, big_fields, "t", time.time(), 1, "delivered"),
)
conn.commit()
conn.close()
ok = apply_landlock_sandbox(
storage_dir=storage,
reticulum_config_dir=storage,
log_dir=storage,
)
if not ok:
print("LANDLOCK_NOT_APPLIED")
sys.exit(2)
DatabaseProvider._instance = None
db = Database(db_path)
# Skip full schema init; only need pressure pragma path + provider.
assert db.apply_memory_pressure_pragmas(True, landlock_active=True)
mode = int(db.provider.connection.execute("PRAGMA temp_store").fetchone()[0])
print("TEMP_MODE", mode)
if mode != 2:
sys.exit(3)
if db.provider.prefer_temp_store_file:
sys.exit(4)
heavy = '''
SELECT m1.id, substr(COALESCE(m1.content,''),1,240) AS content,
CASE WHEN instr(m1.fields,'"image"')>0 THEN 1 ELSE 0 END AS has_image
FROM lxmf_messages m1
INNER JOIN (
SELECT peer_hash, MAX(id) AS max_id FROM lxmf_messages
WHERE peer_hash IS NOT NULL GROUP BY peer_hash
) m2 ON m1.peer_hash=m2.peer_hash AND m1.id=m2.max_id
GROUP BY m1.peer_hash ORDER BY m1.id DESC LIMIT 50
'''
errors = []
def worker(_i):
try:
rows = db.provider.fetchall(heavy)
if not rows:
errors.append("empty")
except Exception as e:
errors.append(str(e))
with ThreadPoolExecutor(max_workers=8) as ex:
list(ex.map(worker, range(24)))
print("PRESSURE_ERRORS", len(errors))
if errors:
print(errors[:3])
sys.exit(5)
sys.exit(0)
"""Tb4b4b4,
Tb4b4b4)
Te6edf3result Tff7b72= Te6edf3subprocessTff7b72.Td2a8ffrunTb4b4b4(
Tb4b4b4[Te6edf3sysTff7b72.Td2a8ffexecutableTb4b4b4, Ta5d6ff"Ta5d6ff-cTa5d6ff"Tb4b4b4, Te6edf3scriptTb4b4b4]Tb4b4b4,
Te6edf3cwdTff7b72=Tffa657strTb4b4b4(Te6edf3PathTb4b4b4(Tff7b72__file__Tb4b4b4)Tff7b72.Td2a8ffresolveTb4b4b4(Tb4b4b4)Tff7b72.Td2a8ffparentsTb4b4b4[T79c0ff2Tb4b4b4]Tb4b4b4)Tb4b4b4,
Te6edf3capture_outputTff7b72=Tff7b72TrueTb4b4b4,
Te6edf3textTff7b72=Tff7b72TrueTb4b4b4,
Te6edf3timeoutTff7b72=T79c0ff60Tb4b4b4,
Te6edf3checkTff7b72=Tff7b72FalseTb4b4b4,
Tb4b4b4)
Tff7b72if Ta5d6ff"Ta5d6ffLANDLOCK_NOT_APPLIEDTa5d6ff" Tff7b72in Te6edf3resultTff7b72.Td2a8ffstdoutTb4b4b4:
Te6edf3pytestTff7b72.Td2a8ffskipTb4b4b4(Ta5d6ff"Ta5d6ffLandlock could not be applied in this environmentTa5d6ff"Tb4b4b4)
Tff7b72assert Te6edf3resultTff7b72.Td2a8ffreturncode Tff7b72== T79c0ff0Tb4b4b4, Tb4b4b4(
Ta5d6fffTa5d6ff"Ta5d6ffstdout=Tffd700{Te6edf3resultTff7b72.Td2a8ffstdoutTffd700!r}Ta5d6ff stderr=Tffd700{Te6edf3resultTff7b72.Td2a8ffstderrTffd700!r}Ta5d6ff code=Tffd700{Te6edf3resultTff7b72.Td2a8ffreturncodeTffd700}Ta5d6ff"
Tb4b4b4)
Tff7b72assert Ta5d6ff"Ta5d6ffTEMP_MODE 2Ta5d6ff" Tff7b72in Te6edf3resultTff7b72.Td2a8ffstdout
Tff7b72assert Ta5d6ff"Ta5d6ffPRESSURE_ERRORS 0Ta5d6ff" Tff7b72in Te6edf3resultTff7b72.Td2a8ffstdout
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────